# -*- coding: utf-8 -*-

# Define here the models for your spider middleware
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/spider-middleware.html
import re
from scrapy import signals
#from scrapy import log
from scrapy.http import Request
#from scrapy.item import BaseItem
from scrapy.utils.request import request_fingerprint
#from xiaohongshu.items import XiaohongshuItem, XiaohongshuGood, XiaohongshuAlbum
import pymongo
import logging
logger = logging.getLogger(__name__)

class XiaohongshuSpiderMiddleware(object):
    # Not all methods need to be defined. If a method is not defined,
    # scrapy acts as if the spider middleware does not modify the
    # passed objects.

    @classmethod
    def from_crawler(cls, crawler):
        # This method is used by Scrapy to create your spiders.
        s = cls()
        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
        return s

    def process_spider_input(response, spider):
        # Called for each response that goes through the spider
        # middleware and into the spider.

        # Should return None or raise an exception.
        return None

    def process_spider_output(response, result, spider):
        # Called with the results returned from the Spider, after
        # it has processed the response.

        # Must return an iterable of Request, dict or Item objects.
        for i in result:
            yield i

    def process_spider_exception(response, exception, spider):
        # Called when a spider or process_spider_input() method
        # (from other spider middleware) raises an exception.

        # Should return either None or an iterable of Response, dict
        # or Item objects.
        pass

    def process_start_requests(start_requests, spider):
        # Called with the start requests of the spider, and works
        # similarly to the process_spider_output() method, except
        # that it doesn’t have a response associated.

        # Must return only requests (not items).
        for r in start_requests:
            yield r

    def spider_opened(self, spider):
        spider.logger.info('Spider opened: %s' % spider.name)

class IgnoreDataBaseItems(object):
    """
    
    """
    collection_item = 'items'
    collection_album = 'albums'
    def __init__(self, mongo_uri, mongo_db):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db
        self.match_related_discoveries = re.compile(".*related_discoveries")
        self.match_items = re.compile(".*discovery/item/")
        self.client = pymongo.MongoClient(self.mongo_uri)
        self.db = self.client[self.mongo_db]
        
    @classmethod
    def from_crawler(cls, crawler):
        return cls(
            mongo_uri=crawler.settings.get('MONGO_URI'),
            mongo_db=crawler.settings.get('MONGO_DATABASE')
        )

    def process_spider_output(self, response, result, spider):
        current_url = response.url
        if self.match_related_discoveries.match(current_url):
            pass
        elif self.match_items.match(current_url):
            item_id = current_url.split("/")[-1]
            if self.db.items.count({"_id":item_id})>0:
                spider.logger.info("Ignoring already visited item: %s" % item_id)
                return []
        return result
        
        
        
class IgnoreVisitedItems(object):
    """
    Middleware to ignore re-visiting item pages if they were already visited
    before. The requests to be filtered by have a meta['filter_visited'] flag
    enabled and optionally define an id to use for identifying them, which
    defaults the request fingerprint, although you'd want to use the item id,
    if you already have it beforehand to make it more robust.
    """
 
    FILTER_VISITED = 'filter_visited'
    VISITED_ID = 'visited_id'
    CONTEXT_KEY = 'visited_ids'
 
    def process_spider_output(self, response, result, spider):
        context = getattr(spider, 'context', {})
        visited_ids = context.setdefault(self.CONTEXT_KEY, {})
        ret = []
        for x in result:
            visited = False
            if isinstance(x, Request):
                if self.FILTER_VISITED in x.meta:
                    visit_id = self._visited_id(x)
                    if visit_id in visited_ids:
                        spider.logger.info("Ignoring already visited: %s" % x.url)
                        visited = True
#            elif isinstance(x, BaseItem):
#                visit_id = self._visited_id(response.request)
#                if visit_id:
#                    visited_ids[visit_id] = True
#                    x['_id'] = visit_id
#                    x['visit_status'] = 'new'
            if visited:
                pass                
                #ret.append(MyItem(visit_id=visit_id, visit_status='old'))
            else:
                ret.append(x)
        return ret
 
    def _visited_id(self, request):
        return request.meta.get(self.VISITED_ID) or request_fingerprint(request)